# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'Yang'
__mtime__ = '2018/10/30'
# code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
            ┃      ☃      ┃
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗━━━┓
                ┃  神兽保佑    ┣┓
                ┃　永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
"""
import numpy as np
import pandas as pd
import re

# with open('51jop_sep.csv', 'w', encoding='utf8')as f_new:
#     with open('51job.csv', 'r', encoding='utf8')as f:
#         while True:
#             line = f.readline()
#             if not line:
#                 break
#             f_new.write(line.replace('；', '::'))


pd.set_option('display.max_columns', 100)
df = pd.read_csv('51job_new_sep.csv', encoding='utf-8', engine='python',
                 names=['jobName_url', 'money', 'company_name', 'company_address', 'job_desc'], delimiter='::')
print(df[df.isnull().values].to_csv('51.job_null.csv'))
df.dropna(inplace=True)
print(df.head(5))
df.jobName_url = df.jobName_url.apply(lambda x: re.sub(',', '_', x))
reg = re.compile('')
df.jobName_url = df.jobName_url.apply(lambda x: re.sub(re.compile('\(|\)|\[|\]|\''), '', x))
print(df.head(5))
df.money = df.money.apply(lambda x: re.sub('\[|\]|\'|万/月', '', x))
df.company_name = df.company_name.apply(lambda x: re.sub('\[|\]|\'', '', x))
df.company_address = df.company_address.apply(lambda x: re.sub('\[|\]|\'', '', x))
df.job_desc = df.job_desc.apply(lambda x: re.sub('\[|\]|\'\ ', '', x))

df = df.astype(str)
no_y = df[df.jobName_url.str.contains('数据')].jobName_url
ret = list(set(no_y) & set(df.jobName_url))
print(len(ret))
df = df[df.jobName_url.isin(ret)]

df.to_csv('51.job_big_data.csv')
